package centroidtc;

import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 *  Handle simple tokenisation tasks relating to terms (words) and categories
 */
public class Tokenizer extends StringTokenizer {

	public static char[] SEPTKARR = { ' ','`','\'','"','_',',','|',
		(new String("\n")).charAt(0),
		(new String("\t")).charAt(0),
		'?', '!', (new String(";")).charAt(0), ':', '<', '>', '{', '}',
		'[', ']', '=', '+', '\\', '%', '$', '*', '&', '(', ')' };
	
	public static String SEPTOKEN = new String(SEPTKARR);

	public Tokenizer(String text) {
		super(text, SEPTOKEN, false);
	}
	
	public boolean isNumeric(String str) {
		boolean ret = false;
		if (str.equalsIgnoreCase("") || str == null) {
			return ret;
		}
		if (str.length() < 2) {
			return ret;
		}
		Pattern p = null;
		Matcher m = null;
		// term is numeric
		p = Pattern.compile("^\\d+\\.?\\d*");
		m = p.matcher(str);
		if (m.find()) {
			ret = true;
			return ret;
		}
		return ret;
	}
	
	/**
	 *  filter some token
	 */
	public String fixEasyWord(String str) {
		String work = null;
		Pattern p = null;
		Matcher m = null;
		// get rid of those non-charactor's symbol
		p = Pattern.compile("^\\W+");
		m = p.matcher(str);
		String s = m.replaceFirst("");
		p = Pattern.compile("\\W+$");
		m = p.matcher(s);
		work = m.replaceFirst("");
		return work;
	}
	
	/**
	 *  filter pre-fix
	 */
	public String fixPreWord(String str) {
		String ret = str;
		// start with 
		Pattern p = Pattern.compile("^\\W+");
		Matcher m = p.matcher(str);
		if (m.find()) {
			ret = m.replaceFirst(str);
		}
		return ret;
	}
	
	/**
	 *  filter post word
	 */
	public String fixPostWord(String str) {
		String ret = str;
		// end with 
		Pattern p = Pattern.compile("\\W+$");
		Matcher m = p.matcher(str);
		if (m.find()) {
			ret = m.replaceFirst(str);
		}
		return ret;
	}

}
